# -*- coding: utf-8 -*-
import scrapy
from scrapy.selector import Selector
from scrapy.http import Request
import json
import re
from doubanMovie.items import DoubanmovieItem

class DoubanmovieSpider(scrapy.Spider):
    name = 'doubanMovie'
    allowed_domains = ['douban.com']
    start_urls = []

    def start_requests(self):
        file_object = open('F:\CrawlMovie\movie_star.txt', 'r')

        try:
            url_head = "https://www.douban.com/j/search?cat=1002&q="
            for line in file_object:
                temp = re.sub(r'\n', '', (url_head + line))
                real_url = temp + '&start=0'
                request = Request(real_url, callback=self.parse)
                request.meta['star_name'] = re.sub(r'\n', '', line)
                request.meta['count'] = 20
                yield request
        finally:
            file_object.close()

    def parse(self, response):
        jsDict = json.loads(response.body)
        jsItems = jsDict['items']
        jsFlag = jsDict['more']
        star_name = response.meta['star_name']
        count = response.meta['count']
        print(count)
        i = 0
        print('......jsflag is:')
        print(jsFlag)
        if jsFlag:
            print('iii')
            temp_url = 'https://www.douban.com/j/search?cat=1002&q='
            temp_name = temp_url + star_name
            real_url = temp_name + '&start=' + str(count)
            print('real_url is:')
            print(real_url)
            count += 20
            request = Request(real_url, callback=self.parse)
            request.meta['star_name'] = star_name
            request.meta['count'] = count
            yield request

        for each in jsItems:
            # selector没有成功提取
            # selector = Selector(each)
            # ss = selector.xpath('//div[@class="result"]/div[@class="pic"]/a/@href')

            # 正则表达式匹配出详情页面的链接
            rep_url = r"(?<=href=\").+?(?=\")"
            temp_link = re.findall(rep_url, each, re.I | re.S | re.M)
            ref_subject = re.findall('subject%[\d|\w]+%', temp_link[0])
            res_subject = re.sub('2F', '', re.sub('subject|%', '', ref_subject[0]))
            base_url = 'https://movie.douban.com/subject/' + str(res_subject)

            pattern = re.compile('\[(.*?)\]', re.S)
            realType = re.findall(pattern, each)
            print(realType[0])
            requiredType = '电影'

            if realType[0] == requiredType:
                print(base_url)
                request = Request(base_url, callback=self.parse_detail)
                request.meta['star_name'] = star_name
                yield request
            i = i + 1

    # 详情页面
    def parse_detail(self, response):
        selector = Selector(response)
        item = DoubanmovieItem()

        star_name = response.meta['star_name']
        # 提取电影名字
        movieName = selector.xpath('//span[@property="v:itemreviewed"]/text()').extract()
        # 提取电影类型
        movieType = selector.xpath('//div[@id="info"]/span[@property="v:genre"]/text()').extract()
        # 提取所属电影的制片国家/地区
        producer_list = selector.xpath('//div[@id="info"]/text()').extract()
        # 上映日期
        temp_time = selector.xpath('//span[@class="year"]/text()').extract()[0]
        release_time = re.findall(r'[\w]+', temp_time)
        # 提取豆瓣评分
        doubanScore = selector.xpath('//strong[@class="ll rating_num"]/text()').extract()
        # 提取评论人数
        commentCount = selector.xpath('//div[@class="rating_sum"]/a/span[@property="v:votes"]/text()').extract()

        film_producer = []
        for temp in producer_list:
            film_producer = re.findall(r'[\w]+', temp)
            if film_producer:
                break

        if commentCount and movieType:
            print('release movie is:')
            item['star_name'] = star_name
            item['movie_name'] = movieName[0]
            item['movie_type'] = movieType
            item['film_producer'] = film_producer
            item['release_time'] = release_time[0]
            item['release_yn'] = 1
            item['douban_score'] = doubanScore[0]
            item['comment_count'] = commentCount[0]
            print(item)
            yield item
        else:
            temp_judge = selector.xpath('//div[@class="rating_sum"]/text()').extract()[0]
            judge = re.findall(r'[\w]+', temp_judge)
            if judge[0] == '尚未上映':
                print('norelease movie is:')
                item['star_name'] = star_name
                item['movie_name'] = movieName[0]
                item['movie_type'] = movieType
                item['film_producer'] = film_producer
                item['release_time'] = release_time[0]
                item['release_yn'] = 0
                item['douban_score'] = 0
                item['comment_count'] = 0
                print(item)
                yield item
            else:
                print('old version should be discarded.')

        # print(film_producer)
        # print(star_name)
        # print(movieName)
        # print(doubanScore)
        # print(commentCount)
        # print(movieType)














